package com.leo.csdnspider.processor;

import com.alibaba.fastjson.JSONObject;
import com.leo.csdnspider.entity.ArchiveInfo;
import org.apache.log4j.Logger;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;

import java.text.SimpleDateFormat;
import java.util.List;


/**
 * @ClassName: ArchiveInfoProcessor
 * @Author: Leo
 * @Description: 归档信息
 * @Date: 9/3/2019 8:12 PM
 */
public class ArchiveInfoProcessor implements PageProcessor {
    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);
    private Logger logger = Logger.getLogger(ArchiveInfoProcessor.class);

    @Override
    public void process(Page page) {
        Html csdn = page.getHtml();

        //归档日期
        List<String> archiveDate = csdn.xpath("//*[@id=\"asideArchive\"]/div/ul/").xpath("/li/a/text()").all();

        //归档数量
        List<String> archiveCount = csdn.xpath("//*[@id=\"asideArchive\"]/div/ul/").xpath("/li/a/span/text()").all();

        //放入Map
        JSONObject jsonObject = new JSONObject();
        for (int i = 0; i < archiveDate.size(); i++) {
            jsonObject.put(archiveDate.get(i).trim(), archiveCount.get(i));
        }

        //userID
        String urlUsername = csdn.xpath("/html/head/link[1]/@href").get().substring(22);

        //爬取时间
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        String date = sdf.format(System.currentTimeMillis());


        ArchiveInfo archiveInfo = new ArchiveInfo(null, urlUsername, jsonObject.toString(), date);
        page.putField("archiveInfo", archiveInfo);
        logger.info("归档信息 -----> " + archiveInfo);
    }

    @Override
    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        Spider spider = Spider.create(new ArchiveInfoProcessor());
        spider.addUrl("https://blog.csdn.net/guolin_blog");
        spider.thread(5);
        spider.setExitWhenComplete(true);
        spider.start();
        spider.stop();
    }
}
